package com.tim.test.download;

import com.alibaba.fastjson.JSON;
import com.gargoylesoftware.htmlunit.BrowserVersion;
import com.gargoylesoftware.htmlunit.NicelyResynchronizingAjaxController;
import com.gargoylesoftware.htmlunit.WebClient;
import com.gargoylesoftware.htmlunit.html.HtmlPage;
import jdk.nashorn.api.scripting.ScriptUtils;
import org.apache.commons.lang3.RandomUtils;
import org.apache.commons.lang3.StringUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.junit.Test;
import org.springframework.web.util.JavaScriptUtils;

import javax.script.ScriptEngine;
import javax.script.ScriptEngineManager;
import javax.script.ScriptException;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.IOException;
import java.nio.charset.Charset;
import java.nio.charset.MalformedInputException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardOpenOption;
import java.util.Arrays;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.stream.Collectors;

/**
 * jQuery选择器规则，https://www.runoob.com/jquery/jquery-ref-selectors.html：
 * 	.选择class，#选择id，[]选择属性，:contains(第)内容包含‘第’
 * 	p,h2；同时选择p、h2
 * 	div.main；同时满足
 * 	div .comment；子孙
 * 	div>p；子级
 * 	div~h1；兄弟
 */
public class BxzwHtmlAnt {
	// 解析地址
	String hrefAddr = "http://www.biqugse.com/101888/";
	// 链接url前缀
	String urlPre = "http://www.biqugse.com";
	// name-href元素父级
	String hrefContainerCss = "#list";
	String hrefElementCss = "a[href]";
	// 内容元素父级
	String contentContainerCss = "#wrapper > div.content_read > div";
	String contentElementCss = "#content";
	// 截断标记。空时不截断
	String filterStr = "无尽的昏迷过后，时宇猛地从床上起身";

	// 文件名
	String fileName = "%s.txt";
	// 目标文件夹
	String dir = "D:/个人资料/书籍/novel/";


	@Test
	public void downloadByIndex() throws Exception {

		/* step1，解析下载地址*/
		Map<String, String> nameHrefMap = getNameHrefMap(hrefAddr, urlPre);

		/* step2，获取内容*/
		StringBuilder builder = getAllContentParallel(nameHrefMap);
//		StringBuilder builder = getAllContentSerial(nameHrefMap);

		/* step3，写入到文件*/
		File dirFile = new File(dir);
		if(!dirFile.exists()){
			dirFile.mkdirs();
		}
		String content = builder.toString()
				.replaceAll("\n+", "\n");
		Files.write(Paths.get(dir,fileName), content.getBytes(StandardCharsets.UTF_8));
		// 输出GB18030编码
		File gbkDirFile = new File(dir+"/GBK");
		if(!gbkDirFile.exists()){
			gbkDirFile.mkdirs();
		}
		Files.write(Paths.get(dir+"/GBK",fileName), content.getBytes(Charset.forName("GB18030")));
//			HttpUtil.downloadFile(v, fileName);
	}

	private Map<String, String> getNameHrefMap(String hrefAddr, String urlPre) throws IOException {
		Document document = Jsoup.connect(hrefAddr).get();
		// 获取文档标题。不改
		Elements title = document.head().select("title");
		if(title!=null){
			String titleText = title.text();
			int index = titleText.indexOf("最新章节");
			titleText = titleText.substring(0, index>0?index:titleText.length());
			fileName = String.format(fileName, titleText).replaceAll("\\s", "");
		}
		// 初筛；获取较大的元素。
		Elements containers = document.body().select(hrefContainerCss);
		// 选中目标元素。
		Elements targetTags = containers.select(hrefElementCss);

		// 筛选目标元素，获取html链接
		Map<String, String> nameHrefMap = new LinkedHashMap<>(128);
		boolean flag = false;
		for(Element targetTag : targetTags){
			// 查找所有子孙级a标签，获取下载链接
			String href = targetTag.attr("href").trim();
			String pageUrl = href.startsWith("http") ? href : urlPre + href;
			String text = targetTag.text().trim();
			// 处理最新章节在前的情况
			if(!flag && (text.startsWith("第一章")||text.startsWith("1"))){
				flag = true;
			}
			if(flag){
				nameHrefMap.put(targetTag.text(), pageUrl);
			}
		}

//		nameHrefMap.forEach((k,v)->{
//			System.out.println(k+","+v);
//		});
		return nameHrefMap;
	}

	private StringBuilder getAllContentSerial(Map<String, String> nameHrefMap) throws Exception {
		StringBuilder builder = new StringBuilder();
		for(Map.Entry<String, String> entry : nameHrefMap.entrySet()) {
			String key=entry.getKey();
			String val=entry.getValue();
			System.out.println(key+"，\t"+val);
			String content = getContent(val);
			if(StringUtils.isBlank(content)){
				throw new RuntimeException(entry.getKey()+" 未获取到内容");
			}
			builder.append(key+"\n")
					.append(content+"\n");
		}
		return builder;
	}

	/*
	 * ConcurrentHashMap；处理并发问题，顺序由遍历nameHrefMap处理
	 * Executors；获取线程池
	 * CountDownLatch；阻塞线程
	 */
	private StringBuilder getAllContentParallel(Map<String, String> nameHrefMap) throws Exception {
		Map<String, String> contentMap = new ConcurrentHashMap<>(4096);
		int threadNum = 50;
		ExecutorService executorService = Executors.newFixedThreadPool(threadNum);
		CountDownLatch countDownLatch = new CountDownLatch(nameHrefMap.size());

		// 获取每个链接的内容
		for(Map.Entry<String, String> entry : nameHrefMap.entrySet()){
			String key=entry.getKey();
			String val=entry.getValue();
			System.out.println(key+"，\t"+val);
			executorService.submit(()->{
				try {
					StringBuilder builder = new StringBuilder();
					String content = getContent(val);
					builder.append(key+"\n")
							.append(content+"\n");
					contentMap.put(key, builder.toString());
				} catch (Exception e) {
					e.printStackTrace();
				}finally {
					countDownLatch.countDown();
				}
			});
		}

		// 按序拼装结果
		countDownLatch.await();// 等待处理完全部链接
		StringBuilder builder = new StringBuilder();
		for(Map.Entry<String, String> entry : nameHrefMap.entrySet()) {
			String content = contentMap.get(entry.getKey());
			if(StringUtils.isBlank(content)){
				throw new RuntimeException(entry.getKey()+" 未获取到内容");
			}
			builder.append(content);
		}
		return builder;
	}

	private String getContent(String href) throws Exception {
		StringBuilder builder = new StringBuilder();
		Document document = Jsoup.connect(href).get();
		// 初筛
		Elements containers = document.body().select(contentContainerCss);
		// 选中目标元素
		Elements targetTags = containers.select(contentElementCss);

		// 获取内容
		for (Element targetTag : targetTags){
			String text = getText(targetTag).trim();
			builder.append(text).append("\n");
		}
		if(StringUtils.isNotBlank(filterStr)){
			return builder.substring(0, Math.max(builder.indexOf(filterStr), builder.length()));
		}
		return builder.toString();
	}

	private String getText(Element child) {
		return child.html()
				.replaceAll("(<br>)+", "\n")
				.replaceAll("\n+", "\n")
				.replaceAll("&nbsp;", " ");

	}

	@Test
	public void downloadByPage() throws Exception {
		// 解析地址
		// 星火2003
		String firstPage = "https://www.bxzw.cc/book/18549/53704475.html";
		// 人族镇守使
//		String firstPage = "https://www.or7.net/or7b/1581911409/43157797.html";
//		String firstPage = "https://www.or7.net/or7b/1582048206/53450520.html";
		// 链接url前缀
		String urlPre = "https://www.bxzw.cc/book/18549/";
		if(StringUtils.isBlank(urlPre)){
			urlPre = firstPage.substring(0, firstPage.lastIndexOf("/")+1);
		}

		File dirFile = new File(dir);
		if(!dirFile.exists()){
			dirFile.mkdirs();
		}

		File file = new File(dir+"/"+pageFileName);
		if(!file.exists()){
			file.createNewFile();
		}
		downloadByPage(firstPage, urlPre);
		System.out.println("SUCCESS");
		if (webClient!=null){
			webClient.close();
		}
	}

	String pageFileName = "pageFileName"+ RandomUtils.nextInt()+".txt";
	String chapterTitle ="";
	int executeJs = 0;
	int skipMultiTag = 0;
	Charset charset = StandardCharsets.UTF_8;
	private Object downloadByPage(String href, String urlPre) throws Exception{
		Thread.sleep(RandomUtils.nextInt(100,2000));
		String contentContainerCss = "#content";
		String contentElementCss = "#htmlContent";
		String titleCss = "h1.readTitle";
		String nextPageCss = "#linkNext";// 下一页
		String nextPageCss2 = "#linkNext";// 下一章；处理下一页、下一章不同

		System.out.println(href);
		Document document;
		if(executeJs==1){
			String html = executeJs(href);
			document = Jsoup.parse(new ByteArrayInputStream(html.getBytes(charset)), charset.name(), "");
		}else {
			document = Jsoup.connect(href).get();
		}
		// 初筛
		Elements containers = document.body().select(contentContainerCss);

		/* step1，获取本页内容，追加写入文档*/
		StringBuilder builder = new StringBuilder();
		if(StringUtils.isNotBlank(titleCss)){
			Elements titleElements = document.body().select(titleCss);
			if(!chapterTitle.equals(titleElements.first().text())){
				chapterTitle = titleElements.first().text();
				builder.append(chapterTitle).append("\n");
			}
		}
		// 选中目标元素
		Elements targetTags = containers.select(contentElementCss);
		// 获取内容
		for (Element targetTag : targetTags){
			if(targetTag.childrenSize()>0 && skipMultiTag==1){
				continue;
			}
			String text = getText(targetTag).trim();
			text = Arrays.stream(text.split("[\r\n]+"))
					.filter(e -> !e.startsWith("笔下中文网 www") && !e.startsWith("<p class"))
					.collect(Collectors.joining("\n"));
			builder.append(text).append("\n");
		}
		String content = builder.toString()
				.replaceAll("\n+", "\n");


		Charset charset1 = document.charset();
		Files.write(Paths.get(dir,pageFileName),
				content.getBytes(charset),
				StandardOpenOption.APPEND
		);


		/* step2，获取下一页链接；无则终止*/
		Elements nextPages = document.body().select(nextPageCss);
		Element nextPage = nextPages.stream()
				.filter(e->{
					String text = e.text();
					return StringUtils.isNotBlank(text) && text.trim().contains("下一");
				}).findFirst().orElse(null);
		if(nextPage!=null){
			String nextPageHref = nextPage.attr("href");
			nextPageHref = nextPageHref.startsWith("http") ? nextPageHref : urlPre + nextPageHref;
			return downloadByPage(nextPageHref, urlPre);
		}else if(StringUtils.isNotBlank(nextPageCss2)){
			Elements nextPages2 = document.body().select(nextPageCss2);
			Element nextPage2 = nextPages2.stream()
					.filter(e->{
						String text = e.text();
						return StringUtils.isNotBlank(text) && text.trim().contains("下一");
					}).findFirst().orElse(null);
			if(nextPage2!=null){
				String nextPageHref = nextPage2.attr("href");
				nextPageHref = nextPageHref.startsWith("http") ? nextPageHref : urlPre + nextPageHref;
				return downloadByPage(nextPageHref, urlPre);
			}
		}
		return null;
	}

	WebClient webClient;
	public String executeJs(String url){
		if(webClient==null){
			webClient = new WebClient(BrowserVersion.CHROME);//新建一个模拟谷歌Chrome浏览器的浏览器客户端对象

			webClient.getOptions().setThrowExceptionOnScriptError(false);//当JS执行出错的时候是否抛出异常, 这里选择不需要
			webClient.getOptions().setThrowExceptionOnFailingStatusCode(false);//当HTTP的状态非200时是否抛出异常, 这里选择不需要
			webClient.getOptions().setActiveXNative(false);//不启用ActiveX
			webClient.getOptions().setCssEnabled(false);//是否启用CSS, 因为不需要展现页面, 所以不需要启用
			webClient.getOptions().setJavaScriptEnabled(true); //很重要，启用JS
			webClient.getOptions().setDownloadImages(false);//不下载图片
			webClient.setAjaxController(new NicelyResynchronizingAjaxController());//很重要，设置支持AJAX

		}
		HtmlPage page = null;
		try {
			page = webClient.getPage(url);//尝试加载上面图片例子给出的网页
		} catch (Exception e) {
			e.printStackTrace();
		}

		webClient.waitForBackgroundJavaScript(20*1000);//异步JS执行需要耗时,所以这里线程要阻塞30秒,等待异步JS执行结束

		charset = page.getCharset();
		String pageXml = page.asXml();//直接将加载完成的页面转换成xml格式的字符串
//		Document parse = Jsoup.parse(pageXml);
//		System.out.println(parse);
		return pageXml;
	}

	@Test
	public void handle() throws IOException {
		String filePath = "F:\\个人资料\\书籍\\novel\\pageFileName1502337489.txt" +
				"";
		String[] filters = new String[]{
				"正在手打中",
				"近期更新缓慢"
		};
		Path path = Paths.get(filePath);
		List<String> list;
		try{
			list = Files.readAllLines(path);
		}catch (MalformedInputException e){
			list = Files.readAllLines(path, Charset.forName("gb2312"));
		}
		String content = list.stream()
				.filter(e -> !StringUtils.startsWithAny(e.trim(),filters))
				.map(String::trim)
				.collect(Collectors.joining("\n"));
		content = content.replaceAll("((<p>)|(</p>)|(<br>)|(<br/>))", "")
				.replaceAll("((<b>)|(</b>))", "")
				.replaceAll("　　", "")
				.replaceAll("・", ".")
				.replaceAll(" {2,}", "  ")
				.replaceAll("单?M", "单芃");
		Files.write(path,
				content.getBytes());
	}
}
